# -*- coding: utf-8 -*-
import scrapy

import re
import json
import logging
import datetime
from copy import copy, deepcopy
from FDC_spider.items import FdcEstateGuidItem, FdcBuildingItem, FdcRoomItem

logger = logging.getLogger(__name__)


class JiujiangSpider(scrapy.Spider):
    name = 'jiujiang'
    allowed_domains = ['zjj-xx.jiujiang.gov.cn']
    start_urls = ['http://zjj-xx.jiujiang.gov.cn/yushouxuke.php']
    permit_li_url = 'http://zjj-xx.jiujiang.gov.cn/RSalesQuery.php?psid={}&bdesc='  # GET 预售许可证列表

    custom_settings = {
        'DOWNLOAD_TIMEOUT': 60,  # 该网站访问延迟较高，设置超时时间为60s
        'RETRY_TIMES': 30,
        'REDIS_HASH_KEY': 'jiujiang_hash',
    }

    def start_requests(self):
        """
        构造POST请求
        按年份进行预售许可证查询
        :return:
        """
        headers = {
            'Referer': self.start_urls[0],
        }
        min_year = 2005  # 经测试，该网站2005年开始才有数据
        max_year = datetime.date.today().year  # 当前年份
        for year in range(min_year, max_year + 1):
            yield scrapy.Request(
                self.permit_li_url.format(year),
                headers=headers,
                meta=dict(year=deepcopy(year), ),
                priority=10,
            )

    def parse(self, response):
        """
        获取预售许可证列表
        :param response:
        :return:
        """
        year = copy(response.meta['year'])
        # 获取预售许可证列表分组
        permit_tr_li = response.xpath("//td[contains(text(),'预售许可证号')]/../following-sibling::tr")
        if permit_tr_li:
            # 遍历，提取项目信息
            for permit_tr in permit_tr_li:
                item_eg = FdcEstateGuidItem()
                item_eg['preSalePermit'] = permit_tr.xpath("./td[2]/a/text()").extract_first()  # 预售许可证
                item_eg['projectName'] = permit_tr.xpath("./td[3]/a/text()").extract_first()  # 项目名称
                item_eg['districtName'] = permit_tr.xpath("./td[4]/a/text()").extract_first()  # 项目区域
                item_eg['projectAddress'] = permit_tr.xpath("./td[5]/a/text()").extract_first()  # 项目地址
                item_eg['preSaleBlockName'] = permit_tr.xpath("./td[6]/a/text()").extract_first()  # 预售楼栋名称
                item_eg['developerName'] = permit_tr.xpath("./td[7]/a/text()").extract_first()  # 开发商
                bd_detail_url = permit_tr.xpath("./td[8]/a/@href").extract_first()  # GET 楼栋详情url
                item_eg['projectId'] = self.get_project_id(bd_detail_url)  # 项目id

                # 获取楼栋详情
                if bd_detail_url:
                    yield response.follow(
                        bd_detail_url,
                        callback=self.parse_building_detail,
                        meta=dict(item_eg=deepcopy(item_eg), ),
                        priority=20,
                    )
                else:
                    logger.error('{} 楼栋详情url提取失败'.format(item_eg['projectName']))
        else:
            logger.warning(f'{year}年，预售许可证列表为空')

    def parse_building_detail(self, response):
        """
        楼栋详情（包含项目信息、房号列表）
        :param response:
        :return:
        """
        item_eg = copy(response.meta['item_eg'])

        # 获取项目详情信息
        project_detail_url = response.xpath("//a[contains(text(),'项目详情')]/@href").extract_first()  # GET 项目详情url
        if project_detail_url:
            # 判断项目详情是否已经爬取
            # 如果未爬取，构造项目详情页请求
            if not self.redis_conn.hexists(self.settings['REDIS_HASH_KEY'], item_eg['projectId']):
                yield response.follow(
                    project_detail_url,
                    callback=self.parse_project_detail,
                    dont_filter=True,  # 对于同一个项目会存在重复请求，可以利用redis做缓存来解决
                    meta=dict(item_eg=deepcopy(item_eg), ),
                    priority=30,
                )
            # 如果已爬取，直接查询并提取数据
            else:
                item_dict: dict = json.loads(
                    self.redis_conn.hget(self.settings['REDIS_HASH_KEY'], item_eg['projectId']))
                # 遍历，对楼盘(项目)字段赋值
                for key, value in item_dict.items():
                    if not item_eg.get(key, None):
                        item_eg[key] = value
        else:
            logger.error('{}-{} 项目详情url提取为空'.format(item_eg['projectName'], item_eg['preSalePermit']))

        # 提取楼栋信息
        item_bd = FdcBuildingItem()
        item_bd['projectName'] = item_eg['projectName']
        item_bd['projectId'] = item_eg['projectId']
        item_bd['preSalePermit'] = item_eg['preSalePermit']
        item_bd['blockName'] = item_eg['preSaleBlockName']
        item_bd['buildingLocation'] = response.xpath(
            "//td[contains(text(),'楼栋座落')]/following-sibling::td[1]/text()").extract_first()  # 楼栋坐落
        item_bd['planningPermit'] = response.xpath(
            "//td[contains(text(),'用地规划')]/following-sibling::td[1]/text()").extract_first()  # 用地规划许可证
        item_bd['constructionPlanningPermit'] = response.xpath(
            "//td[contains(text(),'工程规划')]/following-sibling::td[1]/text()").extract_first()  # 工程规划许可证
        item_bd['constructionPermit'] = response.xpath(
            "//td[contains(text(),'工程施工')]/following-sibling::td[1]/text()").extract_first()  # 施工许可证
        item_bd['certDate'] = response.xpath(
            "//td[contains(text(),'批准预售时间')]/following-sibling::td[1]/text()").extract_first()  # 发证日期
        item_bd['landPermit'] = response.xpath(
            "//td[contains(text(),'国土证号')]/following-sibling::td[1]/text()").extract_first()  # 土地许可证
        item_bd['residenceNum'] = response.xpath(
            "//td[contains(text(),'国土证号')]/../td[4]/text()").extract_first()  # 住宅套数
        item_bd['nonResidenceNum'] = response.xpath(
            "//td[contains(text(),'非住宅套数')]/following-sibling::td[1]/text()").extract_first()  # 非住宅套数
        item_bd['saleableRoomNo'] = response.xpath(
            "//td[contains(text(),'现可售套数')]/following-sibling::td[1]/text()").extract_first()  # 可售套数
        item_bd['soldRoomNo'] = response.xpath(
            "//td[contains(text(),'已售套数')]/following-sibling::td[1]/text()").extract_first()  # 已售套数
        item_bd['residenceAvgPrice'] = response.xpath(
            "//td[contains(text(),'住宅销售均价')]/following-sibling::td[1]/text()").extract_first()  # 住宅均价
        item_bd['nonResidenceAvgPrice'] = response.xpath(
            "//td[contains(text(),'非住宅销售均价')]/following-sibling::td[1]/text()").extract_first()  # 非住宅均价
        item_bd['salesTel'] = response.xpath(
            "//td[contains(text(),'售楼电话')]/following-sibling::td[1]/text()").extract_first()  # 销售电话

        floor_num_list = list()  # 楼层num列表，用于提取楼栋总楼层
        # 获取楼层分组
        floor_tr_li = response.xpath("//h2[contains(text(),'楼盘表')]/../following-sibling::table[2]/tr")
        if floor_tr_li:
            for floor_tr in floor_tr_li:
                floor_num = self.get_floor_num(''.join(floor_tr.xpath("./td[1]/span/text()").extract()))
                floor_num_list.append(floor_num) if isinstance(floor_num, int) and floor_num > 0 else False
                # 获取当前楼层的房号分组
                room_td_li = floor_tr.xpath("./td[2]/table//tr/td/div/..")
                for room_td in room_td_li:
                    item_rm = FdcRoomItem()
                    item_rm['projectName'] = item_bd['projectName']  # 项目名称
                    item_rm['projectId'] = item_bd['projectId']  # 项目id
                    item_rm['preSalePermit'] = item_bd['preSalePermit']  # 预售许可证
                    item_rm['blockName'] = item_bd['blockName']  # 楼栋名字
                    item_rm['roomFloor'] = floor_num  # 房屋楼层
                    room_info_li: list = room_td.xpath("./text()").extract()
                    item_rm['roomNo'], item_rm['buildingArea'], item_rm['innerArea'] = self.get_room_fields(
                        room_info_li)  # 房号、建筑面积、套内面积
                    item_rm['saleStatus'] = room_td.xpath("./div/@title").extract_first()  # 销售状态
                    if item_rm['roomNo']:
                        yield item_rm
                    else:
                        logger.error(f'{room_info_li} 房号字段信息提取出错')

            # 楼栋总楼层
            item_bd['floorTotalNo'] = self.get_floor_total_num(floor_num_list)  # 总层数
            yield item_bd
        else:
            logger.error(
                '{}-{}-{} 楼层分组获取为空'.format(item_bd['projectName'], item_bd['preSalePermit'], item_bd['blockName']))

    def parse_project_detail(self, response):
        """
        项目详情
        :param response:
        :return:
        """
        item_eg = copy(response.meta['item_eg'])
        item_eg['projectUrl'] = response.request.url  # 项目url
        item_eg['projectNature'] = response.xpath(
            "//td[contains(text(),'项目性质')]/following-sibling::td[1]/text()").extract_first()  # 项目性质
        item_eg['estateTotalArea'] = response.xpath(
            "//td[contains(text(),'总建筑面积')]/following-sibling::td[1]/text()").extract_first()  # 建筑总面积
        item_eg['greeningRate'] = response.xpath(
            "//td[contains(text(),'绿化率')]/following-sibling::td[1]/text()").extract_first()  # 绿化率
        item_eg['floorAreaRatio'] = response.xpath(
            "//td[contains(text(),'容积率')]/following-sibling::td[1]/text()").extract_first()  # 容积率
        item_eg['commencementDate'] = response.xpath(
            "//td[contains(text(),'开工日期')]/following-sibling::td[1]/text()").extract_first()  # 开工日期
        item_eg['supervisingUnit'] = response.xpath(
            "//td[contains(text(),'监理单位')]/following-sibling::td[1]/text()").extract_first()  # 监理单位
        item_eg['constructionUnit'] = response.xpath(
            "//td[contains(text(),'施工单位')]/following-sibling::td[1]/text()").extract_first()  # 施工单位
        item_eg['designUnit'] = response.xpath(
            "//td[contains(text(),'设计单位')]/following-sibling::td[1]/text()").extract_first()  # 设计单位
        item_eg['propertyCompany'] = response.xpath(
            "//td[contains(text(),'物业管理')]/following-sibling::td[1]/text()").extract_first()  # 物业公司
        item_eg['salesTel'] = response.xpath(
            "//td[contains(text(),'物业管理')]/following-sibling::td[1]/text()").extract_first()  # 销售电话
        self.redis_conn.hset(self.settings['REDIS_HASH_KEY'], item_eg['projectId'], json.dumps(dict(item_eg)))
        yield item_eg

    @staticmethod
    def get_project_id(_cont: str):
        """
        提取项目id
        :param _cont:
        :return:
        """
        regex = re.compile(r'lpid=(.*?)&')
        try:
            ret = regex.findall(_cont)[0]
            assert ret
        except:
            return
        else:
            return ret

    @staticmethod
    def get_floor_num(_cont: str):
        """
        提取楼层num
        :param _cont:
        :return:
        """
        regex = re.compile(r'-{0,1}\d+', re.S)
        try:
            cont = _cont.strip()
            assert cont
            ret = int(regex.findall(_cont)[0])
        except:
            return _cont
        else:
            return ret

    @staticmethod
    def get_floor_total_num(_cont: list):
        """
        提取总楼层
        :param _cont:
        :return:
        """
        try:
            ret = sorted(_cont, reverse=False)[-1]
            assert ret
        except:
            return
        else:
            return ret

    @staticmethod
    def get_room_fields(_cont: list):
        """
        提取房号字段信息
        :param _cont:
        :return:
        """
        try:
            cont = [i.strip() for i in _cont if i and i.strip()]
            assert len(cont) == 3
            room_no = cont[0]
            building_area = cont[1].split('：')[1]
            inner_area = cont[2].split('：')[1]
        except:
            return None, None, None
        else:
            return room_no, building_area, inner_area
